#!/usr/bin/perl
use LWP::UserAgent;
use Encode;
use HTTP::Headers;
use Digest::SHA1 qw(sha1_hex);

my $keyfile;
my $save_to_pre;
my $max_urls;

if(@ARGV < 2){
	&Useage;
}

for(0..@ARGV){
	$keyfile = @ARGV[$_] if $_ == 0;
	$save_to_pre = @ARGV[$_] if $_ == 1;
	$max_urls = @ARGV[$_] if $_ == 2;
}

if(!defined($max_urls)){
	$max_urls = 5000;
}

if(-e $keyfile){
	open KF, $keyfile or die "$!";
	my $rec_count = 0;
	my $no_result_count = 0;
	my $save_to = $save_to_pre;

	while(<KF>){
		chomp($_);
		my $curr_key = $_;
		$_ = encode("utf-8",decode("gb2312", $_));
		my $curr_key_encode = $_;
		
		#init no_result counter
		$no_result_count = 0;
		print "keyword: $_\n-------------------------------------------------------\n";

		for(1..50){
			my $curr_page = 'http://search.blog.qikoo.com/index.html?kw=' . $curr_key . '&type=&page=' . $_;
			print "$curr_key_encode第$_页...\n";
			
			my $r = &getUrl($curr_page);
			#print encode("utf-8", decode("gbk",$r)), "\n";

			my $no_result = 1;
			while($r =~ /event\);\"><a href=\"(.*?)\" target=\"_blank\"><img/ig){
				print "$1\n";
				if(($rec_count > 0) && ($rec_count % $max_urls == 0)){
					$save_to = $save_to_pre . "_" . ($rec_count + 1) . "_" . ($rec_count + $max_urls);
					print "转换存储文件到：$save_to\n";
				}
				&Output($save_to, $1);
				$rec_count++;
				$no_result = 0;
			}
			if($no_result == 1){
				$no_result_count++;
			}
			last if $no_result_count == 3;
		}
	}
}else{
	print "关键词文件不存在！\n";
}

#使用帮助
sub Useage {
	print "Rss定向爬行器(Qikoo)\n";
	print "crawl.pl <关键词文件> <地址存放文件> [每文件存放地址数目]\n";
	print "by nowa 2007-01-12 at Beijing.\n";

	exit 1;
}

#输出SQL语句到文件
sub Output {
	open SQL, ">>@_[0]" or die "$!";
	print SQL "insert into roboturl (id,key,uri,createdate) values('" . sha1_hex(@_[1]) . "','" . sha1_hex(@_[1]) . "','@_[1]',now()) ;\n";
	close SQL;
}

#获取指定URL的内容
sub getUrl {
	my $ua = LWP::UserAgent->new;
	$ua->agent("nGeter/0.1");
	my $req = HTTP::Request->new(GET => @_);
	my $res = $ua->request($req);
	my $u = "";
	if($res->is_success){
		$u = $res->content;
	}else{
		print $res->status_line, "\n";
	}

	return $u;
}
